package com.chance.cc.crawler.development.scripts.autohome.carSeriesAndConfig;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class AutoHomeCarSeriesAndConfigCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AutoHomeCarSeriesAndConfigCrawlerScript.class);//日志
    private static final String DOMAIN = "autohome";
    private static final String SITE = "carSeriesAndConfig";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //   https://car.autohome.com.cn/javascript/NewSpecCompare.js?20131010
    private static final String HomeRegulars = "https://car.autohome.com.cn/javascript/NewSpecCompare.js\\?20131010";//动态链接
    //    https://club.autohome.com.cn/135
    private static final String CarSeriesRegulars = "https://www.autohome.com.cn/\\d*";//车系链接
    //    https://club.autohome.com.cn/bbs/forum-c-135-1.html#pvareaid=3454552
    private static final String ForumRegulars = "https://club.autohome.com.cn/bbs/forum-c-\\d*-1.html#pvareaid=\\d*";//论坛链接
    //    https://www.autohome.com.cn/135/sale.html#pvareaid=3311673
    private static final String DiscontinuedRegulars = "https://www.autohome.com.cn/\\d*/sale.html#pvareaid=\\d*";//停售链接
    private static final String DiscontinuedTwoRegulars = "https://www.autohome.com.cn/\\d*/sale.html";//停售链接
    //    https://car.autohome.com.cn/config/spec/46839.html#pvareaid=3454495
    private static final String ConfigRegulars = "https://car.autohome.com.cn/config/spec/\\d*.html";//配置链接

    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(CarSeriesRegulars);
        addUrlRegular(ForumRegulars);
        addUrlRegular(DiscontinuedRegulars);
        addUrlRegular(DiscontinuedTwoRegulars);
        addUrlRegular(ConfigRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析详情和在售配置
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ForumRegulars)) {
            forumRequest(crawlerRecord, page, parsedLinks);//解析论坛
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedTwoRegulars)) {
            configRequest(crawlerRecord, page, parsedLinks);//解析停售配置
        }
        return parsedLinks;
    }

    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
            String url = "https://www.autohome.com.cn/" + name.get(i);//得到车系页面
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
                    .copyBizTags()
                    .needWashed(false)
                    .build();
            parsedLinks.add(record);
        }
    }

    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        String forumUrl = null;
        String forumStr = page.getHtml().xpath("//ul[@class=\"channel\"]/li[12]/a/@href").get();
        if (null != forumStr) {
            forumUrl = "https:" + forumStr;  //论坛url
        }
        String koubeiUrl = "https:" + page.getHtml().xpath("//div[@id=\"navTop\"]//a[./text()=\"口碑\"]/@href").get();//口碑url
        String brandName = page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/text()").get();//品牌名
        String brandUrl = "https:" + page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/@href").get();//品牌url
        String subsidiaryBrandName = page.getHtml().xpath("//div[@class=\"athm-sub-nav__car__name\"]/a/text()").get();//子品牌名
        String carSeriesName = page.getHtml().xpath("//h1/a").get();//车系名
        String carSeriesName2 = null;
        if (carSeriesName != null) {
            carSeriesName2 = subsidiaryBrandName + carSeriesName;//车系2
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        HashMap<String, String> seriesMap = new HashMap<>();
        seriesMap.put(AutoVMCommonField.Field_Series_name, carSeriesName);//车系名
        seriesMap.put(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series, seriesMap);//车系
        HashMap<String, String> brandMap = new HashMap<>();
        brandMap.put(AutoVMCommonField.Field_Brand_name, brandName);//品牌名
        brandMap.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Brand, brandMap);//品牌
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Praise_url, koubeiUrl);//口碑
//        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Forum_url, forumUrl);//论坛url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series_Two, carSeriesName2);//二级车系
//        crawlerRecord.tagsCreator().bizTags().addCustomKV("subsidiaryBrandName", subsidiaryBrandName);//子品牌

        if (forumUrl != null) {
            CrawlerRequestRecord forumRecord = CrawlerRequestRecord.builder()//解析论坛
                    .itemPageRequest(crawlerRecord)
                    .recordKey(forumUrl)
                    .httpUrl(forumUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            parsedLinks.add(forumRecord);
        }

        List<String> configUrls = page.getHtml().xpath("//a[./text()=\"配置\"]/@href").all();//所有在售配置
        for (String configUrlStr : configUrls) {
            if (configUrlStr != null && configUrlStr != "") {
                String configId = Pattern.compile("[^0-9]").matcher(configUrlStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
                //https://car.autohome.com.cn/config/spec/46839.html#pvareaid=3454495
                if (configId != null && configId != "") {
                    String configUrl = "https://car.autohome.com.cn/config/spec/" + configId + ".html";
                    CrawlerRequestRecord configurationRecord = CrawlerRequestRecord.builder()//解析在售
                            .itemPageRequest(crawlerRecord)
                            .recordKey(configUrl)
                            .httpUrl(configUrl)
                            .releaseTime(System.currentTimeMillis())
                            .resultLabelTag(article)
                            .needWashed(true)
                            .copyBizTags()
                            .build();
                    parsedLinks.add(configurationRecord);
                }
            }
        }
    }


    private void forumRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String forumUrl = page.getRequest().getUrl();
        String forumName = page.getHtml().xpath("//div[@class=\"bbs-title-name\"]/a").get();//论坛名
        HashMap<String, String> forumMap = new HashMap<>();
        forumMap.put(AutoVMCommonField.Field_Forum_name, forumName);
        forumMap.put(AutoVMCommonField.Field_Forum_url, forumUrl);
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Forum_detail, forumMap);//论坛

        String carSeriesUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AutoVMCommonField.Field_Series_url);
        String articleId = Pattern.compile("[^0-9]").matcher(carSeriesUrl).replaceAll("").trim();//url的关键字
//        https://www.autohome.com.cn/135/sale.html#pvareaid=3311673
        String discontinuedUrl = "https://www.autohome.com.cn/" + articleId + "/sale.html";

        CrawlerRequestRecord saleRecord = CrawlerRequestRecord.builder()//解析在售
                .itemPageRequest(crawlerRecord)
                .recordKey(carSeriesUrl)
                .httpUrl(carSeriesUrl)
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(article)
                .copyBizTags()
                .needWashed(true)
                .notFilterRecord()
                .build();
        saleRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Forum_detail, forumMap);
        parsedLinks.add(saleRecord);

        CrawlerRequestRecord discontinuedRecord = CrawlerRequestRecord.builder()//解析停售
                .itemPageRequest(crawlerRecord)
                .recordKey(discontinuedUrl)
                .httpUrl(discontinuedUrl)
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(article)
                .copyBizTags()
                .needWashed(true)
                .build();
        discontinuedRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Forum_detail, forumMap);
        parsedLinks.add(discontinuedRecord);
    }


    //    解析配置
    private void configRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String url = page.getRequest().getUrl();//获取url

        List<Selectable> nodes = page.getHtml().xpath("//td[@class=\"info_d\"]").nodes();//所有停售款
        for (Selectable node : nodes) {
            String configUrlStr = node.xpath(".//a[./text()=\"参数配置\"]/@href").get();//停售配置
            if (configUrlStr != null && configUrlStr != "") {
                String configId = Pattern.compile("[^0-9]").matcher(configUrlStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
                //https://car.autohome.com.cn/config/spec/46839.html#pvareaid=3454495
                String configUrl = "https://car.autohome.com.cn/config/spec/" + configId + ".html";
                CrawlerRequestRecord configRecord = CrawlerRequestRecord.builder()//解析停售
                        .itemPageRequest(crawlerRecord)
                        .recordKey(configUrl)
                        .httpUrl(configUrl)
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(article)
                        .needWashed(true)
                        .copyBizTags()
                        .build();
                parsedLinks.add(configRecord);
            }
        }
    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(saleWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedTwoRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(discontinuedWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(ConfigRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(configWash(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }

    //    解析在售款
    public List<CrawlerData> saleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        List<Selectable> nodes = page.getHtml().xpath("//div[@class=\"name-param\"]").nodes();//所有在售款

        for (Selectable node : nodes) {
            String saleTitle = node.xpath("./p/a/text()").get();//在售标题名
            String saleUrl = "https://www.autohome.com.cn" + node.xpath("./p/a/@href").get();//在售url
            //https://www.autohome.com.cn/135
            String articleId = Pattern.compile("[^0-9]").matcher(saleUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", domain(), article, articleId))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(carSeriesUrl)
                    .addContentKV(AutoVMCommonField.Field_Model_name, saleTitle)
                    .addContentKV(AutoVMCommonField.Field_Model_url, saleUrl)
                    .addContentKV(AutoVMCommonField.Tag_Is_sale, "1")
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        }
        return crawlerArticleDataList;
    }

    //    解析停售款
    public List<CrawlerData> discontinuedWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Selectable> nodes = page.getHtml().xpath("//td[@class=\"name_d\"]").nodes();//所有停售款

        for (Selectable node : nodes) {
            String discontinuedTitle = node.xpath("./div/a/text()").get();//停售标题名
            String discontinuedUrl = "https://www.autohome.com.cn/" + node.xpath("./div/a/@href").get();//停售url

            String carSeriesUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AutoVMCommonField.Field_Series_url);
            //https://www.autohome.com.cn/135
            String articleId = Pattern.compile("[^0-9]").matcher(carSeriesUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", domain(), article, articleId))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(carSeriesUrl)//carSeriesUrl
                    .addContentKV(AutoVMCommonField.Field_Model_name, discontinuedTitle)
                    .addContentKV(AutoVMCommonField.Field_Model_url, discontinuedUrl)
                    .addContentKV(AutoVMCommonField.Tag_Is_sale, "0")
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        }
        return crawlerArticleDataList;
    }


    //    冲洗配置款
    public List<CrawlerData> configWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Map<String, Map<String, String>>> configList = new ArrayList<>();
        Map<String, Map<String, String>> map = new HashMap<>();
        Map<String, String> configMap = new HashMap<>();
        String configUrl = page.getRequest().getUrl();
        String configId = Pattern.compile("[^0-9]").matcher(configUrl).replaceAll("").trim();//url的关键字
        String configJson = "";
        String optionJson = "";
        String colorJson = "";
        String innerColorJson = "";
        try {
            String config = page.getRawText().split("var config =")[1].split("var option")[0].trim();
            configJson = config.substring(0, config.length() - 1);//70
            String option = page.getRawText().split("var option =")[1].split("var color")[0].trim();
            optionJson = option.substring(0, option.length() - 1);//159
            String color = page.getRawText().split("var color =")[1].split("var innerColor")[0].trim();
            colorJson = color.substring(0, color.length() - 1);//1
            String innerColor = page.getRawText().split("var innerColor =")[1].split("var bag")[0].trim();
            innerColorJson = innerColor.substring(0, innerColor.length() - 1);//1
        } catch (Exception e) {
            log.error(e.getMessage(), e + "该网页可能不存在");
        }
        if (colorJson != null && !"".equals(colorJson)) {
            List<Selectable> configNodes = new Json(configJson).jsonPath($_type + "..paramtypeitems[*]").nodes();
            for (Selectable type : configNodes) {
                String types = new Json(type.get()).jsonPath(".name").get();//分类
                List<Selectable> configNode = new Json(type.get()).jsonPath(".paramitems[*]").nodes();
                configMap = new HashMap<String, String>();
                for (Selectable node : configNode) {
                    String configName = new Json(node.get()).jsonPath(".name").get();//k值
                    String configVal = new Json(node.get()).jsonPath(".valueitems[0].value").get();//v值
                    configMap.put(configName, configVal);
                }
                map.put(types, configMap);
            }

            List<Selectable> optionNodes = new Json(optionJson).jsonPath($_type + "..configtypeitems[*]").nodes();
            for (Selectable type : optionNodes) {
                String types = new Json(type.get()).jsonPath(".name").get();//分类
                List<Selectable> optionNode = new Json(type.get()).jsonPath(".configitems[*]").nodes();
                configMap = new HashMap<String, String>();
                for (Selectable node : optionNode) {
                    String configName = new Json(node.get()).jsonPath(".name").get();//k值
                    String configVal = new Json(node.get()).jsonPath(".valueitems[0].value").get();//v值
                    configMap.put(configName, configVal);
                }
                map.put(types, configMap);
            }

            configMap = new HashMap<String, String>();
            List<Selectable> colorStr = new Json(colorJson).jsonPath($_type + "..specitems[*]").nodes();
            for (Selectable type : colorStr) {
                String id = new Json(type.get()).jsonPath(".specid").get();//分类
                if (configId == id) {
                    List<Selectable> colorNode = new Json(type.get()).jsonPath(".coloritems[*]").nodes();
                    StringBuffer configName = new StringBuffer();//将文本拼接
                    for (Selectable node : colorNode) {
                        List<String> name = new Json(node.get()).jsonPath(".name").all();//k值
//                    String configVal = new Json(node.get()).jsonPath(".value").get();//v值
                        for (String text : name) {
                            configName.append(text).append(" ");
                        }
                    }
                    configMap.put("外观颜色", configName.toString());
                }
            }

            List<Selectable> innerColorStr = new Json(innerColorJson).jsonPath($_type + "..specitems[*]").nodes();
            for (Selectable type : innerColorStr) {
                String id = new Json(type.get()).jsonPath(".specid").get();//分类
                if (configId == id) {
                    List<Selectable> colorNode = new Json(type.get()).jsonPath(".coloritems[*]").nodes();
                    StringBuffer configName = new StringBuffer();//将文本拼接
                    for (Selectable node : colorNode) {
                        List<String> name = new Json(node.get()).jsonPath(".name").all();//k值
//                    String configVal = new Json(node.get()).jsonPath(".value").get();//v值
                        for (String text : name) {
                            configName.append(text).append(" ");
                        }
                    }
                    configMap.put("内饰颜色", configName.toString());
                }
            }
            map.put("选装包", configMap);
            configList.add(map);

            String a = JSONObject.toJSON(configList).toString();
            String config = a.replaceAll("<span class='", "").replaceAll("'></span>", " ").trim();

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", domain(), article, configId))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(configUrl)
                    .flowInPipelineTag("kafka")
                    .addContentKV("config", config)
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);

        }
        return crawlerArticleDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 10) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
